import asyncio
import json
import time
from loguru import logger

from com.arcfox.base.base_spider import ExecuteType
from com.arcfox.util import async_request as requests
from com.arcfox.util.config_parser import Config
from com.arcfox.yiqing.parser.yiqing_parser import YiQingParser
from com.arcfox.yiqing.spider.yiqing_base_spider import YiQingBaseSpider
from com.arcfox.yiqing.task.manager.yiqing_task_manager import TaskManager
from com.arcfox.yiqing.task.yiqing_task import YiQingTask


# http://bmfw.www.gov.cn/yqfxdjcx/risk.html
# https://search.www.gov.cn/search/fw/cateSearch.do?webid=1&category=bmfw&q=
class YiQingDataSpider(YiQingBaseSpider):

    def __init__(self):
        super().__init__()
        self.url = "http://103.66.32.242:8005/zwfwMovePortal/interface/interfaceJson"
        self.parser = YiQingParser()
        self.task = YiQingTask()
        self.config = Config("yiqing.conf")
        self.task_manager = TaskManager()

    async def _pull_task(self):
        return await self.task_manager.pull_task(), ExecuteType.FINISH

    async def _crawl_by_task(self, task):
        await self.init_session()
        if task:
            for city_code in task:
                await self.__crawl_yq_data(city_code)

    async def __crawl_yq_data(self, city_code):
        self.cur_timestamp = str(int(time.time()))
        base_params = self.get_base_params(self.cur_timestamp)
        base_params.update({
            'key': "2CA32596474B4077834CCC191D351839",
            "area_code": city_code
        })
        response = await requests.post(self.session, self.url, headers=self.get_headers(), data=json.dumps(base_params))
        try:
            result_json = response.json()
            logger.info(result_json)
            if result_json['code'] == 0:
                data = result_json['data']
                await self.parser.save_yq_data(data, city_code)
                # 至少停0.3s 否则会返回失败信息
                await asyncio.sleep(0.5)
            else:
                await self.task_manager.add_task([city_code])
        except Exception as e:
            logger.exception(e)


if __name__ == "__main__":
    asyncio.run(YiQingDataSpider().start_crawl())
